{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent1</th>\n",
       "      <th>sent2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>﻿怎么更改花呗手机号码</td>\n",
       "      <td>我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>也开不了花呗，就这样了？完事了</td>\n",
       "      <td>真的嘛？就是花呗付款</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>花呗冻结以后还能开通吗</td>\n",
       "      <td>我的条件可以开通花呗借款吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>如何得知关闭借呗</td>\n",
       "      <td>想永久关闭借呗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>花呗扫码付钱</td>\n",
       "      <td>二维码扫描可以用花呗吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>花呗逾期后不能分期吗</td>\n",
       "      <td>我这个 逾期后还完了 最低还款 后 能分期吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>花呗分期清空</td>\n",
       "      <td>花呗分期查询</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>借呗逾期短信通知</td>\n",
       "      <td>如何购买花呗短信通知</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>借呗即将到期要还的账单还能分期吗</td>\n",
       "      <td>借呗要分期还，是吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>花呗为什么不能支付手机交易</td>\n",
       "      <td>花呗透支了为什么不可以继续用了</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               sent1                           sent2  label\n",
       "1        ﻿怎么更改花呗手机号码  我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号      1\n",
       "2    也开不了花呗，就这样了？完事了                      真的嘛？就是花呗付款      0\n",
       "3        花呗冻结以后还能开通吗                   我的条件可以开通花呗借款吗      0\n",
       "4           如何得知关闭借呗                         想永久关闭借呗      0\n",
       "5             花呗扫码付钱                     二维码扫描可以用花呗吗      0\n",
       "6         花呗逾期后不能分期吗          我这个 逾期后还完了 最低还款 后 能分期吗      0\n",
       "7             花呗分期清空                          花呗分期查询      0\n",
       "8           借呗逾期短信通知                      如何购买花呗短信通知      0\n",
       "9   借呗即将到期要还的账单还能分期吗                       借呗要分期还，是吗      0\n",
       "10     花呗为什么不能支付手机交易                 花呗透支了为什么不可以继续用了      0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data_df = pd.read_csv(\"data/atec_nlp_sim_train_all.csv\", sep=\"\\t\", header=None, \n",
    "                      encoding=\"utf-8-sig\", names=[\"sent1\", \"sent2\", \"label\"])\n",
    "data_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sent1=data_df.sent1.values[:3501]\n",
    "sent2=data_df.sent2.values[:3501]\n",
    "label=data_df.label.values[:3501]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sent1_=sent1[:2501]\n",
    "sent2_=sent2[:2501]\n",
    "label_=label[:2501]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_sent1=sent1[2501:]\n",
    "_sent2=sent2[2501:]\n",
    "_label=label[2501:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data=pd.DataFrame()\n",
    "all_data['id_left']=range(2501)\n",
    "all_data['text_left']=sent1_\n",
    "all_data['id_right']=range(2501)\n",
    "all_data['text_right']=sent2_\n",
    "all_data['label']=label_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(range(1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data=pd.DataFrame()\n",
    "test_data['id_left']=range(1000)\n",
    "test_data['text_left']=_sent1\n",
    "test_data['id_right']=range(1000)\n",
    "test_data['text_right']=_sent2\n",
    "# test_data['label']=_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matchzoo as mz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data(df_data):\n",
    "# \tdf_data = pd.read_csv(data_path, sep='\\t', header=None)\n",
    "# \tdf_data = pd.DataFrame(df_data.values, columns=['id_left', 'text_left', 'id_right', 'text_right', 'label'])\n",
    "\tdf_data = mz.pack(df_data)\n",
    "\treturn df_data\n",
    "\n",
    "train_data = load_data(all_data)\n",
    "# test_data1=load_data(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python3\n",
    "# -*- coding:utf-8 -*-\n",
    "# Author:ChenYuan\n",
    "\n",
    "import matchzoo as mz\n",
    "import pandas as pd\n",
    "import re  # 这个可以不用\n",
    "# from sklearn import preprocessing  # 用于正则化\n",
    "import numpy as np \n",
    "from matchzoo.preprocessors import BasicPreprocessor\n",
    "# 这里有个Basic处理器有问题\n",
    "# preprocessor = BasicPreprocessor(fixed_length_left:int=30, fixed_length_right:int=30)\n",
    "preprocessor=BasicPreprocessor(15,15)\n",
    "# preprocessor=mz.preprocessors.CDSSMPreprocessor()\n",
    "                                                  # 定义一个数据处理器\n",
    "#     ，有四种处理器，Basic是通用的、基础的数据处理器，可看官方文档，这里不做解说\n",
    "                                        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|█| 2000/2000 [00:00<00:00, 5639\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|█| 2000/2000 [00:00<00:00, 639\n",
      "Processing text_right with append: 100%|███████████████████████████████████████| 2000/2000 [00:00<00:00, 181870.78it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|███████████████████████████████| 2000/2000 [00:00<00:00, 142923.48it/s]\n",
      "Processing text_right with transform: 100%|████████████████████████████████████| 2000/2000 [00:00<00:00, 181945.73it/s]\n",
      "Processing text_left with extend: 100%|████████████████████████████████████████| 2000/2000 [00:00<00:00, 400277.14it/s]\n",
      "Processing text_right with extend: 100%|███████████████████████████████████████| 2000/2000 [00:00<00:00, 250062.84it/s]\n",
      "Building Vocabulary from a datapack.: 100%|████████████████████████████████████| 2132/2132 [00:00<00:00, 711679.76it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|█| 2000/2000 [00:00<00:00, 5767\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|█| 2000/2000 [00:00<00:00, 662\n",
      "Processing text_right with transform: 100%|████████████████████████████████████| 2000/2000 [00:00<00:00, 333689.01it/s]\n",
      "Processing text_left with transform: 100%|█████████████████████████████████████| 2000/2000 [00:00<00:00, 153973.09it/s]\n",
      "Processing text_right with transform: 100%|████████████████████████████████████| 2000/2000 [00:00<00:00, 286017.53it/s]\n",
      "Processing length_left with len: 100%|█████████████████████████████████████████| 2000/2000 [00:00<00:00, 400525.59it/s]\n",
      "Processing length_right with len: 100%|████████████████████████████████████████| 2000/2000 [00:00<00:00, 500453.88it/s]\n",
      "Processing text_left with transform: 100%|██████████████████████████████████████| 2000/2000 [00:00<00:00, 60641.56it/s]\n",
      "Processing text_right with transform: 100%|█████████████████████████████████████| 2000/2000 [00:00<00:00, 76973.12it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|█| 501/501 [00:00<00:00, 5063.4\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|█| 501/501 [00:00<00:00, 5509.\n",
      "Processing text_right with transform: 100%|██████████████████████████████████████| 501/501 [00:00<00:00, 125513.46it/s]\n",
      "Processing text_left with transform: 100%|███████████████████████████████████████| 501/501 [00:00<00:00, 167198.15it/s]\n",
      "Processing text_right with transform: 100%|██████████████████████████████████████| 501/501 [00:00<00:00, 167184.84it/s]\n",
      "Processing length_left with len: 100%|███████████████████████████████████████████| 501/501 [00:00<00:00, 100245.51it/s]\n",
      "Processing length_right with len: 100%|██████████████████████████████████████████| 501/501 [00:00<00:00, 100274.21it/s]\n",
      "Processing text_left with transform: 100%|████████████████████████████████████████| 501/501 [00:00<00:00, 50128.73it/s]\n",
      "Processing text_right with transform: 100%|███████████████████████████████████████| 501/501 [00:00<00:00, 62670.63it/s]\n"
     ]
    }
   ],
   "source": [
    "train_dev_split = int(len(train_data) * 0.8)  # 验证集占训练数据的0.1\n",
    "train = train_data[:train_dev_split]\n",
    "dev = train_data[train_dev_split:]\n",
    "train_pack_processed = preprocessor.fit_transform(train) \n",
    "# 其实就是做了一个字符转id操作，所以对于中文文本，不需要分词\n",
    "dev_pack_processed = preprocessor.transform(dev)  \n",
    "# test_pack_processed = preprocessor.transform(test_data)\n",
    "train_data_generator = mz.DataGenerator(train_pack_processed\n",
    "                                        , batch_size=32\n",
    "                                        , shuffle=True)  # 训练数据生成器\n",
    "\n",
    "# test_x, test_y = test_pack_processed.unpack()\n",
    "dev_x, dev_y = dev_pack_processed.unpack()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DenseBaseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_6\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)     (None, 30)           0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 3)            93          concatenate_6[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 3)            12          dense_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 3)            12          dense_27[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_29 (Dense)                (None, 64)           256         dense_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_30 (Dense)                (None, 1)            65          dense_29[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 438\n",
      "Trainable params: 438\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 0s 148us/step - loss: 8.4868\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 0s 59us/step - loss: 1.0970\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 0s 65us/step - loss: 0.3701\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 0s 64us/step - loss: 0.2099\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 0s 61us/step - loss: 0.1757\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "### 准备数据，数据在源码中有，不确定在pip安装的是否存在\n",
    "### train_raw是matchzoo中自定的数据格式\tmatchzoo.data_pack.data_pack.DataPack\n",
    "# train_raw = mz.datasets.toy.load_data(stage='train', task=task)\n",
    "# test_raw = mz.datasets.toy.load_data(stage='test', task=task)\n",
    "\n",
    "### 数据预处理，BasicPreprocessor为指定预处理的方式，在预处理中包含了两步\n",
    "# ：fit,transform\n",
    "### fit将收集一些有用的信息到preprocessor.context中\n",
    "# ，不会对输入DataPack进行处理\n",
    "### transformer 不会改变context、DataPack,他将重新生成转变后的DataPack.\n",
    "### 在transformer过程中，包含了Tokenize => Lowercase => PuncRemoval等过程\n",
    "# ，这个过程在方法中应该是可以自定义的\n",
    "\n",
    "# preprocessor = mz.preprocessors.BasicPreprocessor()\n",
    "# preprocessor.fit(train_raw)  ## init preprocessor inner state.\n",
    "# train_processed = preprocessor.transform(train_raw)\n",
    "# test_processed = preprocessor.transform(test_raw)\n",
    "\n",
    "### 创建模型以及修改参数（可以使用mz.models.list_available()查看可用的模型列表）\n",
    "model = mz.models.DenseBaseline()\n",
    "model.params['task'] = task\n",
    "model.params['mlp_num_units'] = 3\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/DenseBaseline-model.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# ArcI "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_5\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 300)      629700      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 15, 32)       28832       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 15, 32)       28832       embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1D)  (None, 7, 32)        0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1D)  (None, 3, 32)        0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 224)          0           max_pooling1d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 96)           0           max_pooling1d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 320)          0           flatten_3[0][0]                  \n",
      "                                                                 flatten_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 320)          0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 64)           20544       dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 32)           2080        dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 1)            33          dense_17[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 710,021\n",
      "Trainable params: 710,021\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 3s 2ms/step - loss: 0.1616\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 3s 1ms/step - loss: 0.1578\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 3s 1ms/step - loss: 0.1450\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 3s 1ms/step - loss: 0.0372\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 3s 1ms/step - loss: 0.0241\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "### 准备数据，数据在源码中有，不确定在pip安装的是否存在\n",
    "### train_raw是matchzoo中自定的数据格式\tmatchzoo.data_pack.data_pack.DataPack\n",
    "# train_raw = mz.datasets.toy.load_data(stage='train', task=task)\n",
    "# test_raw = mz.datasets.toy.load_data(stage='test', task=task)\n",
    "\n",
    "### 数据预处理，BasicPreprocessor为指定预处理的方式，在预处理中包含了两步\n",
    "# ：fit,transform\n",
    "### fit将收集一些有用的信息到preprocessor.context中\n",
    "# ，不会对输入DataPack进行处理\n",
    "### transformer 不会改变context、DataPack,他将重新生成转变后的DataPack.\n",
    "### 在transformer过程中，包含了Tokenize => Lowercase => PuncRemoval等过程\n",
    "# ，这个过程在方法中应该是可以自定义的\n",
    "\n",
    "# preprocessor = mz.preprocessors.BasicPreprocessor()\n",
    "# preprocessor.fit(train_raw)  ## init preprocessor inner state.\n",
    "# train_processed = preprocessor.transform(train_raw)\n",
    "# test_processed = preprocessor.transform(test_raw)\n",
    "\n",
    "### 创建模型以及修改参数（可以使用mz.models.list_available()查看可用的模型列表）\n",
    "# model = mz.models.DenseBaseline()\n",
    "# model.params['task'] = task\n",
    "# model.params['mlp_num_units'] = 3\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "# >>> model = ANMM()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.build()\n",
    "\n",
    "        \n",
    "model =mz.models.ArcI()\n",
    "model.params['num_blocks'] = 1\n",
    "model.params['left_filters'] = [32]\n",
    "model.params['right_filters'] = [32]\n",
    "model.params['left_kernel_sizes'] = [3]\n",
    "model.params['right_kernel_sizes'] = [3]\n",
    "model.params['left_pool_sizes'] = [2]\n",
    "model.params['right_pool_sizes'] = [4]\n",
    "model.params['conv_activation_func'] = 'relu'\n",
    "model.params['mlp_num_layers'] = 1\n",
    "model.params['mlp_num_units'] = 64\n",
    "model.params['mlp_num_fan_out'] = 32\n",
    "model.params['mlp_activation_func'] = 'relu'\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/ANMM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ConvKNRM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_6\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 300)      629700      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 15, 128)      38528       embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 15, 128)      76928       embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, 15, 128)      115328      embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dot_5 (Dot)                     (None, 15, 15)       0           conv1d_5[0][0]                   \n",
      "                                                                 conv1d_5[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_6 (Dot)                     (None, 15, 15)       0           conv1d_5[0][0]                   \n",
      "                                                                 conv1d_6[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_7 (Dot)                     (None, 15, 15)       0           conv1d_5[0][0]                   \n",
      "                                                                 conv1d_7[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_8 (Dot)                     (None, 15, 15)       0           conv1d_6[0][0]                   \n",
      "                                                                 conv1d_5[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_9 (Dot)                     (None, 15, 15)       0           conv1d_6[0][0]                   \n",
      "                                                                 conv1d_6[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_10 (Dot)                    (None, 15, 15)       0           conv1d_6[0][0]                   \n",
      "                                                                 conv1d_7[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_11 (Dot)                    (None, 15, 15)       0           conv1d_7[0][0]                   \n",
      "                                                                 conv1d_5[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_12 (Dot)                    (None, 15, 15)       0           conv1d_7[0][0]                   \n",
      "                                                                 conv1d_6[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_13 (Dot)                    (None, 15, 15)       0           conv1d_7[0][0]                   \n",
      "                                                                 conv1d_7[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 15, 15)       0           dot_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 15, 15)       0           dot_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_50 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_52 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_54 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_56 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_58 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_60 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_62 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_64 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_66 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_68 (Activation)      (None, 15, 15)       0           dot_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_70 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_72 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_74 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_76 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_78 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_80 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_82 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_84 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_86 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_88 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_90 (Activation)      (None, 15, 15)       0           dot_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_92 (Activation)      (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_94 (Activation)      (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_96 (Activation)      (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_98 (Activation)      (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_100 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_102 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_104 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_106 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_108 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_110 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_112 (Activation)     (None, 15, 15)       0           dot_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_114 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_116 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_118 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_120 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_122 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_124 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_126 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_128 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_130 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_132 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_134 (Activation)     (None, 15, 15)       0           dot_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_136 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_138 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_140 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_142 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_144 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_146 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_148 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_150 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_152 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_154 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_156 (Activation)     (None, 15, 15)       0           dot_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_158 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_160 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_162 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_164 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_166 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_168 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_170 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_172 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_174 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_176 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_178 (Activation)     (None, 15, 15)       0           dot_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_180 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_182 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_184 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_186 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_188 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_190 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_192 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_194 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_196 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_198 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_200 (Activation)     (None, 15, 15)       0           dot_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 15)           0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 15)           0           activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 15)           0           activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, 15)           0           activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_12 (Lambda)              (None, 15)           0           activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_14 (Lambda)              (None, 15)           0           activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_16 (Lambda)              (None, 15)           0           activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_18 (Lambda)              (None, 15)           0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_20 (Lambda)              (None, 15)           0           activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_22 (Lambda)              (None, 15)           0           activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_24 (Lambda)              (None, 15)           0           activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_26 (Lambda)              (None, 15)           0           activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_28 (Lambda)              (None, 15)           0           activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_30 (Lambda)              (None, 15)           0           activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_32 (Lambda)              (None, 15)           0           activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_34 (Lambda)              (None, 15)           0           activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_36 (Lambda)              (None, 15)           0           activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_38 (Lambda)              (None, 15)           0           activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_40 (Lambda)              (None, 15)           0           activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_42 (Lambda)              (None, 15)           0           activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_44 (Lambda)              (None, 15)           0           activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_46 (Lambda)              (None, 15)           0           activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_48 (Lambda)              (None, 15)           0           activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_50 (Lambda)              (None, 15)           0           activation_50[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_52 (Lambda)              (None, 15)           0           activation_52[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_54 (Lambda)              (None, 15)           0           activation_54[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_56 (Lambda)              (None, 15)           0           activation_56[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_58 (Lambda)              (None, 15)           0           activation_58[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_60 (Lambda)              (None, 15)           0           activation_60[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_62 (Lambda)              (None, 15)           0           activation_62[0][0]              \n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lambda_64 (Lambda)              (None, 15)           0           activation_64[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_66 (Lambda)              (None, 15)           0           activation_66[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_68 (Lambda)              (None, 15)           0           activation_68[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_70 (Lambda)              (None, 15)           0           activation_70[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_72 (Lambda)              (None, 15)           0           activation_72[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_74 (Lambda)              (None, 15)           0           activation_74[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_76 (Lambda)              (None, 15)           0           activation_76[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_78 (Lambda)              (None, 15)           0           activation_78[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_80 (Lambda)              (None, 15)           0           activation_80[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_82 (Lambda)              (None, 15)           0           activation_82[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_84 (Lambda)              (None, 15)           0           activation_84[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_86 (Lambda)              (None, 15)           0           activation_86[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_88 (Lambda)              (None, 15)           0           activation_88[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_90 (Lambda)              (None, 15)           0           activation_90[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_92 (Lambda)              (None, 15)           0           activation_92[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_94 (Lambda)              (None, 15)           0           activation_94[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_96 (Lambda)              (None, 15)           0           activation_96[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_98 (Lambda)              (None, 15)           0           activation_98[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_100 (Lambda)             (None, 15)           0           activation_100[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_102 (Lambda)             (None, 15)           0           activation_102[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_104 (Lambda)             (None, 15)           0           activation_104[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_106 (Lambda)             (None, 15)           0           activation_106[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_108 (Lambda)             (None, 15)           0           activation_108[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_110 (Lambda)             (None, 15)           0           activation_110[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_112 (Lambda)             (None, 15)           0           activation_112[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_114 (Lambda)             (None, 15)           0           activation_114[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_116 (Lambda)             (None, 15)           0           activation_116[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_118 (Lambda)             (None, 15)           0           activation_118[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_120 (Lambda)             (None, 15)           0           activation_120[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_122 (Lambda)             (None, 15)           0           activation_122[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_124 (Lambda)             (None, 15)           0           activation_124[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_126 (Lambda)             (None, 15)           0           activation_126[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_128 (Lambda)             (None, 15)           0           activation_128[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_130 (Lambda)             (None, 15)           0           activation_130[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_132 (Lambda)             (None, 15)           0           activation_132[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_134 (Lambda)             (None, 15)           0           activation_134[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_136 (Lambda)             (None, 15)           0           activation_136[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_138 (Lambda)             (None, 15)           0           activation_138[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_140 (Lambda)             (None, 15)           0           activation_140[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_142 (Lambda)             (None, 15)           0           activation_142[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_144 (Lambda)             (None, 15)           0           activation_144[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_146 (Lambda)             (None, 15)           0           activation_146[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_148 (Lambda)             (None, 15)           0           activation_148[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_150 (Lambda)             (None, 15)           0           activation_150[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_152 (Lambda)             (None, 15)           0           activation_152[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_154 (Lambda)             (None, 15)           0           activation_154[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_156 (Lambda)             (None, 15)           0           activation_156[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_158 (Lambda)             (None, 15)           0           activation_158[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_160 (Lambda)             (None, 15)           0           activation_160[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_162 (Lambda)             (None, 15)           0           activation_162[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_164 (Lambda)             (None, 15)           0           activation_164[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_166 (Lambda)             (None, 15)           0           activation_166[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_168 (Lambda)             (None, 15)           0           activation_168[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_170 (Lambda)             (None, 15)           0           activation_170[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_172 (Lambda)             (None, 15)           0           activation_172[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_174 (Lambda)             (None, 15)           0           activation_174[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_176 (Lambda)             (None, 15)           0           activation_176[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_178 (Lambda)             (None, 15)           0           activation_178[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_180 (Lambda)             (None, 15)           0           activation_180[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_182 (Lambda)             (None, 15)           0           activation_182[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_184 (Lambda)             (None, 15)           0           activation_184[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_186 (Lambda)             (None, 15)           0           activation_186[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_188 (Lambda)             (None, 15)           0           activation_188[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_190 (Lambda)             (None, 15)           0           activation_190[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_192 (Lambda)             (None, 15)           0           activation_192[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_194 (Lambda)             (None, 15)           0           activation_194[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_196 (Lambda)             (None, 15)           0           activation_196[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_198 (Lambda)             (None, 15)           0           activation_198[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_200 (Lambda)             (None, 15)           0           activation_200[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 15)           0           lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 15)           0           lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 15)           0           lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 15)           0           lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 15)           0           lambda_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 15)           0           lambda_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 15)           0           lambda_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 15)           0           lambda_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 15)           0           lambda_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 15)           0           lambda_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 15)           0           lambda_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 15)           0           lambda_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 15)           0           lambda_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 15)           0           lambda_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 15)           0           lambda_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 15)           0           lambda_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 15)           0           lambda_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 15)           0           lambda_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 15)           0           lambda_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 15)           0           lambda_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 15)           0           lambda_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 15)           0           lambda_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 15)           0           lambda_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_51 (Activation)      (None, 15)           0           lambda_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_53 (Activation)      (None, 15)           0           lambda_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_55 (Activation)      (None, 15)           0           lambda_54[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_57 (Activation)      (None, 15)           0           lambda_56[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_59 (Activation)      (None, 15)           0           lambda_58[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_61 (Activation)      (None, 15)           0           lambda_60[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_63 (Activation)      (None, 15)           0           lambda_62[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_65 (Activation)      (None, 15)           0           lambda_64[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_67 (Activation)      (None, 15)           0           lambda_66[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_69 (Activation)      (None, 15)           0           lambda_68[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_71 (Activation)      (None, 15)           0           lambda_70[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_73 (Activation)      (None, 15)           0           lambda_72[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_75 (Activation)      (None, 15)           0           lambda_74[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_77 (Activation)      (None, 15)           0           lambda_76[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_79 (Activation)      (None, 15)           0           lambda_78[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_81 (Activation)      (None, 15)           0           lambda_80[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_83 (Activation)      (None, 15)           0           lambda_82[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_85 (Activation)      (None, 15)           0           lambda_84[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_87 (Activation)      (None, 15)           0           lambda_86[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_89 (Activation)      (None, 15)           0           lambda_88[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_91 (Activation)      (None, 15)           0           lambda_90[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_93 (Activation)      (None, 15)           0           lambda_92[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_95 (Activation)      (None, 15)           0           lambda_94[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_97 (Activation)      (None, 15)           0           lambda_96[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_99 (Activation)      (None, 15)           0           lambda_98[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_101 (Activation)     (None, 15)           0           lambda_100[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_103 (Activation)     (None, 15)           0           lambda_102[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_105 (Activation)     (None, 15)           0           lambda_104[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_107 (Activation)     (None, 15)           0           lambda_106[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_109 (Activation)     (None, 15)           0           lambda_108[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_111 (Activation)     (None, 15)           0           lambda_110[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_113 (Activation)     (None, 15)           0           lambda_112[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_115 (Activation)     (None, 15)           0           lambda_114[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_117 (Activation)     (None, 15)           0           lambda_116[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_119 (Activation)     (None, 15)           0           lambda_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_121 (Activation)     (None, 15)           0           lambda_120[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_123 (Activation)     (None, 15)           0           lambda_122[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_125 (Activation)     (None, 15)           0           lambda_124[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_127 (Activation)     (None, 15)           0           lambda_126[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_129 (Activation)     (None, 15)           0           lambda_128[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_131 (Activation)     (None, 15)           0           lambda_130[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_133 (Activation)     (None, 15)           0           lambda_132[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_135 (Activation)     (None, 15)           0           lambda_134[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_137 (Activation)     (None, 15)           0           lambda_136[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_139 (Activation)     (None, 15)           0           lambda_138[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_141 (Activation)     (None, 15)           0           lambda_140[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_143 (Activation)     (None, 15)           0           lambda_142[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_145 (Activation)     (None, 15)           0           lambda_144[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_147 (Activation)     (None, 15)           0           lambda_146[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_149 (Activation)     (None, 15)           0           lambda_148[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_151 (Activation)     (None, 15)           0           lambda_150[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_153 (Activation)     (None, 15)           0           lambda_152[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_155 (Activation)     (None, 15)           0           lambda_154[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_157 (Activation)     (None, 15)           0           lambda_156[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_159 (Activation)     (None, 15)           0           lambda_158[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_161 (Activation)     (None, 15)           0           lambda_160[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_163 (Activation)     (None, 15)           0           lambda_162[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_165 (Activation)     (None, 15)           0           lambda_164[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_167 (Activation)     (None, 15)           0           lambda_166[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_169 (Activation)     (None, 15)           0           lambda_168[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_171 (Activation)     (None, 15)           0           lambda_170[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_173 (Activation)     (None, 15)           0           lambda_172[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_175 (Activation)     (None, 15)           0           lambda_174[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_177 (Activation)     (None, 15)           0           lambda_176[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_179 (Activation)     (None, 15)           0           lambda_178[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_181 (Activation)     (None, 15)           0           lambda_180[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_183 (Activation)     (None, 15)           0           lambda_182[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_185 (Activation)     (None, 15)           0           lambda_184[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_187 (Activation)     (None, 15)           0           lambda_186[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_189 (Activation)     (None, 15)           0           lambda_188[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_191 (Activation)     (None, 15)           0           lambda_190[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_193 (Activation)     (None, 15)           0           lambda_192[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_195 (Activation)     (None, 15)           0           lambda_194[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_197 (Activation)     (None, 15)           0           lambda_196[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_199 (Activation)     (None, 15)           0           lambda_198[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_201 (Activation)     (None, 15)           0           lambda_200[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None,)              0           activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None,)              0           activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None,)              0           activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None,)              0           activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_13 (Lambda)              (None,)              0           activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_15 (Lambda)              (None,)              0           activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_17 (Lambda)              (None,)              0           activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_19 (Lambda)              (None,)              0           activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_21 (Lambda)              (None,)              0           activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_23 (Lambda)              (None,)              0           activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_25 (Lambda)              (None,)              0           activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_27 (Lambda)              (None,)              0           activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_29 (Lambda)              (None,)              0           activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_31 (Lambda)              (None,)              0           activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_33 (Lambda)              (None,)              0           activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_35 (Lambda)              (None,)              0           activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_37 (Lambda)              (None,)              0           activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_39 (Lambda)              (None,)              0           activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_41 (Lambda)              (None,)              0           activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_43 (Lambda)              (None,)              0           activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_45 (Lambda)              (None,)              0           activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_47 (Lambda)              (None,)              0           activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_49 (Lambda)              (None,)              0           activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_51 (Lambda)              (None,)              0           activation_51[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_53 (Lambda)              (None,)              0           activation_53[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_55 (Lambda)              (None,)              0           activation_55[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_57 (Lambda)              (None,)              0           activation_57[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_59 (Lambda)              (None,)              0           activation_59[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_61 (Lambda)              (None,)              0           activation_61[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_63 (Lambda)              (None,)              0           activation_63[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_65 (Lambda)              (None,)              0           activation_65[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_67 (Lambda)              (None,)              0           activation_67[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_69 (Lambda)              (None,)              0           activation_69[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_71 (Lambda)              (None,)              0           activation_71[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_73 (Lambda)              (None,)              0           activation_73[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_75 (Lambda)              (None,)              0           activation_75[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_77 (Lambda)              (None,)              0           activation_77[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_79 (Lambda)              (None,)              0           activation_79[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_81 (Lambda)              (None,)              0           activation_81[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_83 (Lambda)              (None,)              0           activation_83[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_85 (Lambda)              (None,)              0           activation_85[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_87 (Lambda)              (None,)              0           activation_87[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_89 (Lambda)              (None,)              0           activation_89[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_91 (Lambda)              (None,)              0           activation_91[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_93 (Lambda)              (None,)              0           activation_93[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_95 (Lambda)              (None,)              0           activation_95[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_97 (Lambda)              (None,)              0           activation_97[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_99 (Lambda)              (None,)              0           activation_99[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_101 (Lambda)             (None,)              0           activation_101[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_103 (Lambda)             (None,)              0           activation_103[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_105 (Lambda)             (None,)              0           activation_105[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_107 (Lambda)             (None,)              0           activation_107[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_109 (Lambda)             (None,)              0           activation_109[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_111 (Lambda)             (None,)              0           activation_111[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_113 (Lambda)             (None,)              0           activation_113[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_115 (Lambda)             (None,)              0           activation_115[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_117 (Lambda)             (None,)              0           activation_117[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_119 (Lambda)             (None,)              0           activation_119[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_121 (Lambda)             (None,)              0           activation_121[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_123 (Lambda)             (None,)              0           activation_123[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_125 (Lambda)             (None,)              0           activation_125[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_127 (Lambda)             (None,)              0           activation_127[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_129 (Lambda)             (None,)              0           activation_129[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_131 (Lambda)             (None,)              0           activation_131[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_133 (Lambda)             (None,)              0           activation_133[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_135 (Lambda)             (None,)              0           activation_135[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_137 (Lambda)             (None,)              0           activation_137[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_139 (Lambda)             (None,)              0           activation_139[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_141 (Lambda)             (None,)              0           activation_141[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_143 (Lambda)             (None,)              0           activation_143[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_145 (Lambda)             (None,)              0           activation_145[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_147 (Lambda)             (None,)              0           activation_147[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_149 (Lambda)             (None,)              0           activation_149[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_151 (Lambda)             (None,)              0           activation_151[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_153 (Lambda)             (None,)              0           activation_153[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_155 (Lambda)             (None,)              0           activation_155[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_157 (Lambda)             (None,)              0           activation_157[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_159 (Lambda)             (None,)              0           activation_159[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_161 (Lambda)             (None,)              0           activation_161[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_163 (Lambda)             (None,)              0           activation_163[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_165 (Lambda)             (None,)              0           activation_165[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_167 (Lambda)             (None,)              0           activation_167[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_169 (Lambda)             (None,)              0           activation_169[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_171 (Lambda)             (None,)              0           activation_171[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_173 (Lambda)             (None,)              0           activation_173[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_175 (Lambda)             (None,)              0           activation_175[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_177 (Lambda)             (None,)              0           activation_177[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_179 (Lambda)             (None,)              0           activation_179[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_181 (Lambda)             (None,)              0           activation_181[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_183 (Lambda)             (None,)              0           activation_183[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_185 (Lambda)             (None,)              0           activation_185[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_187 (Lambda)             (None,)              0           activation_187[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_189 (Lambda)             (None,)              0           activation_189[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_191 (Lambda)             (None,)              0           activation_191[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_193 (Lambda)             (None,)              0           activation_193[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_195 (Lambda)             (None,)              0           activation_195[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_197 (Lambda)             (None,)              0           activation_197[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_199 (Lambda)             (None,)              0           activation_199[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_201 (Lambda)             (None,)              0           activation_201[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_202 (Lambda)             (None, 99)           0           lambda_5[0][0]                   \n",
      "                                                                 lambda_7[0][0]                   \n",
      "                                                                 lambda_9[0][0]                   \n",
      "                                                                 lambda_11[0][0]                  \n",
      "                                                                 lambda_13[0][0]                  \n",
      "                                                                 lambda_15[0][0]                  \n",
      "                                                                 lambda_17[0][0]                  \n",
      "                                                                 lambda_19[0][0]                  \n",
      "                                                                 lambda_21[0][0]                  \n",
      "                                                                 lambda_23[0][0]                  \n",
      "                                                                 lambda_25[0][0]                  \n",
      "                                                                 lambda_27[0][0]                  \n",
      "                                                                 lambda_29[0][0]                  \n",
      "                                                                 lambda_31[0][0]                  \n",
      "                                                                 lambda_33[0][0]                  \n",
      "                                                                 lambda_35[0][0]                  \n",
      "                                                                 lambda_37[0][0]                  \n",
      "                                                                 lambda_39[0][0]                  \n",
      "                                                                 lambda_41[0][0]                  \n",
      "                                                                 lambda_43[0][0]                  \n",
      "                                                                 lambda_45[0][0]                  \n",
      "                                                                 lambda_47[0][0]                  \n",
      "                                                                 lambda_49[0][0]                  \n",
      "                                                                 lambda_51[0][0]                  \n",
      "                                                                 lambda_53[0][0]                  \n",
      "                                                                 lambda_55[0][0]                  \n",
      "                                                                 lambda_57[0][0]                  \n",
      "                                                                 lambda_59[0][0]                  \n",
      "                                                                 lambda_61[0][0]                  \n",
      "                                                                 lambda_63[0][0]                  \n",
      "                                                                 lambda_65[0][0]                  \n",
      "                                                                 lambda_67[0][0]                  \n",
      "                                                                 lambda_69[0][0]                  \n",
      "                                                                 lambda_71[0][0]                  \n",
      "                                                                 lambda_73[0][0]                  \n",
      "                                                                 lambda_75[0][0]                  \n",
      "                                                                 lambda_77[0][0]                  \n",
      "                                                                 lambda_79[0][0]                  \n",
      "                                                                 lambda_81[0][0]                  \n",
      "                                                                 lambda_83[0][0]                  \n",
      "                                                                 lambda_85[0][0]                  \n",
      "                                                                 lambda_87[0][0]                  \n",
      "                                                                 lambda_89[0][0]                  \n",
      "                                                                 lambda_91[0][0]                  \n",
      "                                                                 lambda_93[0][0]                  \n",
      "                                                                 lambda_95[0][0]                  \n",
      "                                                                 lambda_97[0][0]                  \n",
      "                                                                 lambda_99[0][0]                  \n",
      "                                                                 lambda_101[0][0]                 \n",
      "                                                                 lambda_103[0][0]                 \n",
      "                                                                 lambda_105[0][0]                 \n",
      "                                                                 lambda_107[0][0]                 \n",
      "                                                                 lambda_109[0][0]                 \n",
      "                                                                 lambda_111[0][0]                 \n",
      "                                                                 lambda_113[0][0]                 \n",
      "                                                                 lambda_115[0][0]                 \n",
      "                                                                 lambda_117[0][0]                 \n",
      "                                                                 lambda_119[0][0]                 \n",
      "                                                                 lambda_121[0][0]                 \n",
      "                                                                 lambda_123[0][0]                 \n",
      "                                                                 lambda_125[0][0]                 \n",
      "                                                                 lambda_127[0][0]                 \n",
      "                                                                 lambda_129[0][0]                 \n",
      "                                                                 lambda_131[0][0]                 \n",
      "                                                                 lambda_133[0][0]                 \n",
      "                                                                 lambda_135[0][0]                 \n",
      "                                                                 lambda_137[0][0]                 \n",
      "                                                                 lambda_139[0][0]                 \n",
      "                                                                 lambda_141[0][0]                 \n",
      "                                                                 lambda_143[0][0]                 \n",
      "                                                                 lambda_145[0][0]                 \n",
      "                                                                 lambda_147[0][0]                 \n",
      "                                                                 lambda_149[0][0]                 \n",
      "                                                                 lambda_151[0][0]                 \n",
      "                                                                 lambda_153[0][0]                 \n",
      "                                                                 lambda_155[0][0]                 \n",
      "                                                                 lambda_157[0][0]                 \n",
      "                                                                 lambda_159[0][0]                 \n",
      "                                                                 lambda_161[0][0]                 \n",
      "                                                                 lambda_163[0][0]                 \n",
      "                                                                 lambda_165[0][0]                 \n",
      "                                                                 lambda_167[0][0]                 \n",
      "                                                                 lambda_169[0][0]                 \n",
      "                                                                 lambda_171[0][0]                 \n",
      "                                                                 lambda_173[0][0]                 \n",
      "                                                                 lambda_175[0][0]                 \n",
      "                                                                 lambda_177[0][0]                 \n",
      "                                                                 lambda_179[0][0]                 \n",
      "                                                                 lambda_181[0][0]                 \n",
      "                                                                 lambda_183[0][0]                 \n",
      "                                                                 lambda_185[0][0]                 \n",
      "                                                                 lambda_187[0][0]                 \n",
      "                                                                 lambda_189[0][0]                 \n",
      "                                                                 lambda_191[0][0]                 \n",
      "                                                                 lambda_193[0][0]                 \n",
      "                                                                 lambda_195[0][0]                 \n",
      "                                                                 lambda_197[0][0]                 \n",
      "                                                                 lambda_199[0][0]                 \n",
      "                                                                 lambda_201[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 1)            100         lambda_202[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 860,584\n",
      "Trainable params: 860,584\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000/2000 [==============================] - 12s 6ms/step - loss: 4.2023\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 5s 3ms/step - loss: 0.5019\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 6s 3ms/step - loss: 0.1633\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 6s 3ms/step - loss: 0.1223\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 6s 3ms/step - loss: 0.0795\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "### 准备数据，数据在源码中有，不确定在pip安装的是否存在\n",
    "### train_raw是matchzoo中自定的数据格式\tmatchzoo.data_pack.data_pack.DataPack\n",
    "# train_raw = mz.datasets.toy.load_data(stage='train', task=task)\n",
    "# test_raw = mz.datasets.toy.load_data(stage='test', task=task)\n",
    "\n",
    "### 数据预处理，BasicPreprocessor为指定预处理的方式，在预处理中包含了两步\n",
    "# ：fit,transform\n",
    "### fit将收集一些有用的信息到preprocessor.context中\n",
    "# ，不会对输入DataPack进行处理\n",
    "### transformer 不会改变context、DataPack,他将重新生成转变后的DataPack.\n",
    "### 在transformer过程中，包含了Tokenize => Lowercase => PuncRemoval等过程\n",
    "# ，这个过程在方法中应该是可以自定义的\n",
    "\n",
    "# preprocessor = mz.preprocessors.BasicPreprocessor()\n",
    "# preprocessor.fit(train_raw)  ## init preprocessor inner state.\n",
    "# train_processed = preprocessor.transform(train_raw)\n",
    "# test_processed = preprocessor.transform(test_raw)\n",
    "\n",
    "### 创建模型以及修改参数（可以使用mz.models.list_available()查看可用的模型列表）\n",
    "# model = mz.models.DenseBaseline()\n",
    "# model.params['task'] = task\n",
    "# model.params['mlp_num_units'] = 3\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "# >>> model = ANMM()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.build()\n",
    "\n",
    "\n",
    "model = mz.models.ConvKNRM()\n",
    "model.params['embedding_input_dim'] = 10000\n",
    "model.params['embedding_output_dim'] = 300\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['filters'] = 128\n",
    "model.params['conv_activation_func'] = 'tanh'\n",
    "model.params['max_ngram'] = 3\n",
    "model.params['use_crossmatch'] = True\n",
    "model.params['kernel_num'] = 11\n",
    "model.params['sigma'] = 0.1\n",
    "model.params['exact_sigma'] = 0.001\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "# model =mz.models.DRMM()\n",
    "# model.params['mlp_num_layers'] = 1\n",
    "# model.params['mlp_num_units'] = 5\n",
    "# model.params['mlp_num_fan_out'] = 1\n",
    "# model.params['mlp_activation_func'] = 'tanh'\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "        \n",
    "# model =mz.models.ArcI()\n",
    "# model.params['num_blocks'] = 1\n",
    "# model.params['left_filters'] = [32]\n",
    "# model.params['right_filters'] = [32]\n",
    "# model.params['left_kernel_sizes'] = [3]\n",
    "# model.params['right_kernel_sizes'] = [3]\n",
    "# model.params['left_pool_sizes'] = [2]\n",
    "# model.params['right_pool_sizes'] = [4]\n",
    "# model.params['conv_activation_func'] = 'relu'\n",
    "# model.params['mlp_num_layers'] = 1\n",
    "# model.params['mlp_num_units'] = 64\n",
    "# model.params['mlp_num_fan_out'] = 32\n",
    "# model.params['mlp_activation_func'] = 'relu'\n",
    "# model.params['dropout_rate'] = 0.5\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/ConvKNRM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.models.naive.Naive,\n",
       " matchzoo.models.dssm.DSSM,\n",
       " matchzoo.models.cdssm.CDSSM,\n",
       " matchzoo.models.dense_baseline.DenseBaseline,\n",
       " matchzoo.models.arci.ArcI,\n",
       " matchzoo.models.arcii.ArcII,\n",
       " matchzoo.models.match_pyramid.MatchPyramid,\n",
       " matchzoo.models.knrm.KNRM,\n",
       " matchzoo.models.duet.DUET,\n",
       " matchzoo.models.drmmtks.DRMMTKS,\n",
       " matchzoo.models.drmm.DRMM,\n",
       " matchzoo.models.anmm.ANMM,\n",
       " matchzoo.models.mvlstm.MVLSTM,\n",
       " matchzoo.contrib.models.match_lstm.MatchLSTM,\n",
       " matchzoo.contrib.models.match_srnn.MatchSRNN,\n",
       " matchzoo.contrib.models.hbmp.HBMP,\n",
       " matchzoo.contrib.models.esim.ESIM,\n",
       " matchzoo.contrib.models.bimpm.BiMPM,\n",
       " matchzoo.contrib.models.diin.DIIN,\n",
       " matchzoo.models.conv_knrm.ConvKNRM]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.models.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DRMMTKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_10\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 100)      209900      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_27 (Dot)                    (None, 15, 15)       0           embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_32 (Dense)                (None, 15, 1)        100         embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_404 (Lambda)             (None, 15, 15)       0           dot_27[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "attention_mask (Lambda)         (None, 15, 1)        0           dense_32[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_33 (Dense)                (None, 15, 5)        80          lambda_404[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "attention_probs (Lambda)        (None, 15, 1)        0           attention_mask[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_34 (Dense)                (None, 15, 1)        6           dense_33[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_28 (Dot)                    (None, 1, 1)         0           attention_probs[0][0]            \n",
      "                                                                 dense_34[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 1)            0           dot_28[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_35 (Dense)                (None, 1)            2           flatten_7[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 210,088\n",
      "Trainable params: 210,088\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 1s 523us/step - loss: 0.1690\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 1s 319us/step - loss: 0.1534\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 1s 323us/step - loss: 0.1217\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 1s 358us/step - loss: 0.0538\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 1s 297us/step - loss: 0.0271\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.models.ConvKNRM()\n",
    "# model.params['embedding_input_dim'] = 10000\n",
    "# model.params['embedding_output_dim'] = 300\n",
    "# model.params['embedding_trainable'] = True\n",
    "# model.params['filters'] = 128\n",
    "# model.params['conv_activation_func'] = 'tanh'\n",
    "# model.params['max_ngram'] = 3\n",
    "# model.params['use_crossmatch'] = True\n",
    "# model.params['kernel_num'] = 11\n",
    "# model.params['sigma'] = 0.1\n",
    "# model.params['exact_sigma'] = 0.001\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "model =mz.models.DRMMTKS()\n",
    "model.params['embedding_input_dim'] = 10000\n",
    "model.params['embedding_output_dim'] = 100\n",
    "model.params['top_k'] = 20\n",
    "model.params['mlp_num_layers'] = 1\n",
    "model.params['mlp_num_units'] = 5\n",
    "model.params['mlp_num_fan_out'] = 1\n",
    "model.params['mlp_activation_func'] = 'tanh'\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.build()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "        \n",
    "# model =mz.models.ArcI()\n",
    "# model.params['num_blocks'] = 1\n",
    "# model.params['left_filters'] = [32]\n",
    "# model.params['right_filters'] = [32]\n",
    "# model.params['left_kernel_sizes'] = [3]\n",
    "# model.params['right_kernel_sizes'] = [3]\n",
    "# model.params['left_pool_sizes'] = [2]\n",
    "# model.params['right_pool_sizes'] = [4]\n",
    "# model.params['conv_activation_func'] = 'relu'\n",
    "# model.params['mlp_num_layers'] = 1\n",
    "# model.params['mlp_num_units'] = 64\n",
    "# model.params['mlp_num_fan_out'] = 32\n",
    "# model.params['mlp_activation_func'] = 'relu'\n",
    "# model.params['dropout_rate'] = 0.5\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/DRMMTKS.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DUET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_11\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 300)      629700      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_12 (Conv1D)              (None, 15, 32)       28832       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 15, 32)       0           conv1d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_13 (Conv1D)              (None, 15, 32)       28832       embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1D)  (None, 1, 32)        0           dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 15, 32)       0           conv1d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "reshape_8 (Reshape)             (None, 32)           0           max_pooling1d_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_6 (MaxPooling1D)  (None, 3, 32)        0           dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_405 (Lambda)             (None, 15, 15)       0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_39 (Dense)                (None, 32)           1056        reshape_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_14 (Conv1D)              (None, 3, 32)        1056        max_pooling1d_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_11 (Conv1D)              (None, 15, 32)       7232        lambda_405[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_406 (Lambda)             (None, 1, 32)        0           dense_39[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 3, 32)        0           conv1d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 15, 32)       0           conv1d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_407 (Lambda)             (None, 3, 32)        0           lambda_406[0][0]                 \n",
      "                                                                 dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_7 (Reshape)             (None, 480)          0           dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "reshape_9 (Reshape)             (None, 96)           0           lambda_407[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_36 (Dense)                (None, 64)           30784       reshape_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_40 (Dense)                (None, 64)           6208        reshape_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_37 (Dense)                (None, 32)           2080        dense_36[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_41 (Dense)                (None, 32)           2080        dense_40[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 32)           0           dense_37[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 32)           0           dense_41[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_38 (Dense)                (None, 1)            33          dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_42 (Dense)                (None, 1)            33          dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 1)            0           dense_38[0][0]                   \n",
      "                                                                 dense_42[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_43 (Dense)                (None, 1)            2           add_1[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 737,928\n",
      "Trainable params: 737,928\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 5s 2ms/step - loss: 0.2327\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 3s 2ms/step - loss: 0.1600\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 3s 1ms/step - loss: 0.1573\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 3s 2ms/step - loss: 0.1337\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 3s 1ms/step - loss: 0.0733\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.models.ConvKNRM()\n",
    "model = mz.models.DUET()\n",
    "model.params['embedding_input_dim'] = 1000\n",
    "model.params['embedding_output_dim'] = 300\n",
    "model.params['lm_filters'] = 32\n",
    "model.params['lm_hidden_sizes'] = [64, 32]\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.params['dm_filters'] = 32\n",
    "model.params['dm_kernel_size'] = 3\n",
    "model.params['dm_d_mpool'] = 4\n",
    "model.params['dm_hidden_sizes'] = [64, 32]\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "        \n",
    "# model =mz.models.ArcI()\n",
    "# model.params['num_blocks'] = 1\n",
    "# model.params['left_filters'] = [32]\n",
    "# model.params['right_filters'] = [32]\n",
    "# model.params['left_kernel_sizes'] = [3]\n",
    "# model.params['right_kernel_sizes'] = [3]\n",
    "# model.params['left_pool_sizes'] = [2]\n",
    "# model.params['right_pool_sizes'] = [4]\n",
    "# model.params['conv_activation_func'] = 'relu'\n",
    "# model.params['mlp_num_layers'] = 1\n",
    "# model.params['mlp_num_units'] = 64\n",
    "# model.params['mlp_num_fan_out'] = 32\n",
    "# model.params['mlp_activation_func'] = 'relu'\n",
    "# model.params['dropout_rate'] = 0.5\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/DUET.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ESIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_12\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 10)       20990       text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            multiple             0           embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "                                                                 dense_44[0][0]                   \n",
      "                                                                 dense_44[1][0]                   \n",
      "                                                                 dense_45[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_408 (Lambda)             (None, 15)           0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 15, 600)      746400      dropout_12[0][0]                 \n",
      "                                                                 dropout_12[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_409 (Lambda)             (None, 15, 1)        0           lambda_408[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_410 (Lambda)             (None, 15, 1)        0           lambda_408[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 15, 600)      0           bidirectional_1[0][0]            \n",
      "                                                                 lambda_409[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 15, 600)      0           bidirectional_1[1][0]            \n",
      "                                                                 lambda_410[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_411 (Lambda)             (None, 15, 1)        0           lambda_408[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_412 (Lambda)             (None, 1, 15)        0           lambda_408[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_29 (Dot)                    (None, 15, 15)       0           multiply_1[0][0]                 \n",
      "                                                                 multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 15, 15)       0           lambda_411[0][0]                 \n",
      "                                                                 lambda_412[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 15, 15)       0           dot_29[0][0]                     \n",
      "                                                                 multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "atten_mask (Lambda)             (None, 15, 15)       0           dot_29[0][0]                     \n",
      "                                                                 multiply_3[0][0]                 \n",
      "                                                                 permute_1[0][0]                  \n",
      "                                                                 permute_1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "softmax_1 (Softmax)             (None, 15, 15)       0           atten_mask[0][0]                 \n",
      "                                                                 atten_mask[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_30 (Dot)                    (None, 15, 600)      0           softmax_1[0][0]                  \n",
      "                                                                 multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_31 (Dot)                    (None, 15, 600)      0           softmax_1[1][0]                  \n",
      "                                                                 multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "subtract_1 (Subtract)           (None, 15, 600)      0           multiply_1[0][0]                 \n",
      "                                                                 dot_30[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 15, 600)      0           multiply_1[0][0]                 \n",
      "                                                                 dot_30[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "subtract_2 (Subtract)           (None, 15, 600)      0           multiply_2[0][0]                 \n",
      "                                                                 dot_31[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 15, 600)      0           multiply_2[0][0]                 \n",
      "                                                                 dot_31[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 15, 2400)     0           multiply_1[0][0]                 \n",
      "                                                                 dot_30[0][0]                     \n",
      "                                                                 subtract_1[0][0]                 \n",
      "                                                                 multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 15, 2400)     0           multiply_2[0][0]                 \n",
      "                                                                 dot_31[0][0]                     \n",
      "                                                                 subtract_2[0][0]                 \n",
      "                                                                 multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_44 (Dense)                (None, 15, 300)      720300      concatenate_3[0][0]              \n",
      "                                                                 concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 15, 600)      1442400     dropout_12[2][0]                 \n",
      "                                                                 dropout_12[3][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_413 (Lambda)             (None, 15, 1)        0           lambda_408[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_415 (Lambda)             (None, 15, 1)        0           lambda_408[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_417 (Lambda)             (None, 15, 1)        0           lambda_408[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_419 (Lambda)             (None, 15, 1)        0           lambda_408[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 15, 600)      0           bidirectional_2[0][0]            \n",
      "                                                                 lambda_413[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 15, 600)      0           bidirectional_2[0][0]            \n",
      "                                                                 lambda_415[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 15, 600)      0           bidirectional_2[1][0]            \n",
      "                                                                 lambda_417[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 15, 600)      0           bidirectional_2[1][0]            \n",
      "                                                                 lambda_419[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_414 (Lambda)             (None, 600)          0           multiply_6[0][0]                 \n",
      "                                                                 lambda_413[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_416 (Lambda)             (None, 600)          0           multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_418 (Lambda)             (None, 600)          0           multiply_8[0][0]                 \n",
      "                                                                 lambda_417[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_420 (Lambda)             (None, 600)          0           multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 1200)         0           lambda_414[0][0]                 \n",
      "                                                                 lambda_416[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)     (None, 1200)         0           lambda_418[0][0]                 \n",
      "                                                                 lambda_420[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)     (None, 2400)         0           concatenate_5[0][0]              \n",
      "                                                                 concatenate_6[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_45 (Dense)                (None, 300)          720300      concatenate_7[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_46 (Dense)                (None, 1)            301         dropout_12[4][0]                 \n",
      "==================================================================================================\n",
      "Total params: 3,650,691\n",
      "Trainable params: 3,629,701\n",
      "Non-trainable params: 20,990\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 67s 34ms/step - loss: nan\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 61s 31ms/step - loss: nan\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: nan\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 70s 35ms/step - loss: nan\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: nan\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.models.ConvKNRM()\n",
    "model = mz.contrib.models.ESIM()\n",
    "# task = classification_task = mz.tasks.Classification(num_classes=2)\n",
    "model.params['task'] = task\n",
    "model.params['input_shapes'] = [(15, ), (15, )]\n",
    "model.params['lstm_dim'] = 300\n",
    "model.params['mlp_num_units'] = 300\n",
    "model.params['embedding_input_dim'] =  5000\n",
    "model.params['embedding_output_dim'] =  10\n",
    "model.params['embedding_trainable'] = False\n",
    "model.params['mlp_num_layers'] = 0\n",
    "model.params['mlp_num_fan_out'] = 300\n",
    "model.params['mlp_activation_func'] = 'tanh'\n",
    "model.params['mask_value'] = 0\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.params['optimizer'] = K.optimizers.Adam(lr=4e-4)\n",
    "model.guess_and_fill_missing_params()\n",
    "# model.build()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "        \n",
    "# model =mz.models.ArcI()\n",
    "# model.params['num_blocks'] = 1\n",
    "# model.params['left_filters'] = [32]\n",
    "# model.params['right_filters'] = [32]\n",
    "# model.params['left_kernel_sizes'] = [3]\n",
    "# model.params['right_kernel_sizes'] = [3]\n",
    "# model.params['left_pool_sizes'] = [2]\n",
    "# model.params['right_pool_sizes'] = [4]\n",
    "# model.params['conv_activation_func'] = 'relu'\n",
    "# model.params['mlp_num_layers'] = 1\n",
    "# model.params['mlp_num_units'] = 64\n",
    "# model.params['mlp_num_fan_out'] = 32\n",
    "# model.params['mlp_activation_func'] = 'relu'\n",
    "# model.params['dropout_rate'] = 0.5\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/ESIM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.models.naive.Naive,\n",
       " matchzoo.models.dssm.DSSM,\n",
       " matchzoo.models.cdssm.CDSSM,\n",
       " matchzoo.models.dense_baseline.DenseBaseline,\n",
       " matchzoo.models.arci.ArcI,\n",
       " matchzoo.models.arcii.ArcII,\n",
       " matchzoo.models.match_pyramid.MatchPyramid,\n",
       " matchzoo.models.knrm.KNRM,\n",
       " matchzoo.models.duet.DUET,\n",
       " matchzoo.models.drmmtks.DRMMTKS,\n",
       " matchzoo.models.drmm.DRMM,\n",
       " matchzoo.models.anmm.ANMM,\n",
       " matchzoo.models.mvlstm.MVLSTM,\n",
       " matchzoo.contrib.models.match_lstm.MatchLSTM,\n",
       " matchzoo.contrib.models.match_srnn.MatchSRNN,\n",
       " matchzoo.contrib.models.hbmp.HBMP,\n",
       " matchzoo.contrib.models.esim.ESIM,\n",
       " matchzoo.contrib.models.bimpm.BiMPM,\n",
       " matchzoo.contrib.models.diin.DIIN,\n",
       " matchzoo.models.conv_knrm.ConvKNRM]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.models.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HBMP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_13\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 100)      209900      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) [(None, 15, 10), (No 4240        embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_6 (Bidirectional) [(None, 15, 10), (No 4240        embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional) [(None, 15, 10), (No 4240        embedding[0][0]                  \n",
      "                                                                 bidirectional_3[0][1]            \n",
      "                                                                 bidirectional_3[0][2]            \n",
      "                                                                 bidirectional_3[0][3]            \n",
      "                                                                 bidirectional_3[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_7 (Bidirectional) [(None, 15, 10), (No 4240        embedding[1][0]                  \n",
      "                                                                 bidirectional_6[0][1]            \n",
      "                                                                 bidirectional_6[0][2]            \n",
      "                                                                 bidirectional_6[0][3]            \n",
      "                                                                 bidirectional_6[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_5 (Bidirectional) [(None, 15, 10), (No 4240        embedding[0][0]                  \n",
      "                                                                 bidirectional_4[0][1]            \n",
      "                                                                 bidirectional_4[0][2]            \n",
      "                                                                 bidirectional_4[0][3]            \n",
      "                                                                 bidirectional_4[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_8 (Bidirectional) [(None, 15, 10), (No 4240        embedding[1][0]                  \n",
      "                                                                 bidirectional_7[0][1]            \n",
      "                                                                 bidirectional_7[0][2]            \n",
      "                                                                 bidirectional_7[0][3]            \n",
      "                                                                 bidirectional_7[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 10)           0           bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 10)           0           bidirectional_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 10)           0           bidirectional_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_4 (GlobalM (None, 10)           0           bidirectional_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_5 (GlobalM (None, 10)           0           bidirectional_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_6 (GlobalM (None, 10)           0           bidirectional_8[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)     (None, 30)           0           global_max_pooling1d_1[0][0]     \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_9 (Concatenate)     (None, 30)           0           global_max_pooling1d_4[0][0]     \n",
      "                                                                 global_max_pooling1d_5[0][0]     \n",
      "                                                                 global_max_pooling1d_6[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "subtract_3 (Subtract)           (None, 30)           0           concatenate_8[0][0]              \n",
      "                                                                 concatenate_9[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lambda_421 (Lambda)             (None, 30)           0           subtract_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_10 (Multiply)          (None, 30)           0           concatenate_8[0][0]              \n",
      "                                                                 concatenate_9[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)    (None, 120)          0           concatenate_8[0][0]              \n",
      "                                                                 concatenate_9[0][0]              \n",
      "                                                                 lambda_421[0][0]                 \n",
      "                                                                 multiply_10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 120)          0           concatenate_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_47 (Dense)                (None, 10)           1210        dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "leaky_re_lu_1 (LeakyReLU)       (None, 10)           0           dense_47[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, 10)           0           leaky_re_lu_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_48 (Dense)                (None, 10)           110         dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)       (None, 10)           0           dense_48[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_49 (Dense)                (None, 1)            11          leaky_re_lu_2[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 236,671\n",
      "Trainable params: 236,671\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 14s 7ms/step - loss: 0.1649\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 6s 3ms/step - loss: 0.1561\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 7s 4ms/step - loss: 0.1424\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 8s 4ms/step - loss: 0.0337\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 8s 4ms/step - loss: 0.0142\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "model = mz.contrib.models.HBMP()\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.params['embedding_input_dim'] = 200\n",
    "model.params['embedding_output_dim'] = 100\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['alpha'] = 0.1\n",
    "model.params['mlp_num_layers'] = 3\n",
    "model.params['mlp_num_units'] = [10, 10]\n",
    "model.params['lstm_num_units'] = 5\n",
    "model.params['dropout_rate'] = 0.1\n",
    "# model.build()\n",
    "\n",
    "# model.build()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "        \n",
    "# model =mz.models.ArcI()\n",
    "# model.params['num_blocks'] = 1\n",
    "# model.params['left_filters'] = [32]\n",
    "# model.params['right_filters'] = [32]\n",
    "# model.params['left_kernel_sizes'] = [3]\n",
    "# model.params['right_kernel_sizes'] = [3]\n",
    "# model.params['left_pool_sizes'] = [2]\n",
    "# model.params['right_pool_sizes'] = [4]\n",
    "# model.params['conv_activation_func'] = 'relu'\n",
    "# model.params['mlp_num_layers'] = 1\n",
    "# model.params['mlp_num_units'] = 64\n",
    "# model.params['mlp_num_fan_out'] = 32\n",
    "# model.params['mlp_activation_func'] = 'relu'\n",
    "# model.params['dropout_rate'] = 0.5\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params.update(preprocessor.context)\n",
    "# model.params.completed()\n",
    "# model.build()\n",
    "# model.compile()\n",
    "# model.backend.summary()\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/HBMP.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNRM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_14\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 10)       20990       text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_32 (Dot)                    (None, 15, 15)       0           embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_400 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_402 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_404 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_406 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_408 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_410 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_412 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_414 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_416 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_418 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_420 (Activation)     (None, 15, 15)       0           dot_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_422 (Lambda)             (None, 15)           0           activation_400[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_424 (Lambda)             (None, 15)           0           activation_402[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_426 (Lambda)             (None, 15)           0           activation_404[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_428 (Lambda)             (None, 15)           0           activation_406[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_430 (Lambda)             (None, 15)           0           activation_408[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_432 (Lambda)             (None, 15)           0           activation_410[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_434 (Lambda)             (None, 15)           0           activation_412[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_436 (Lambda)             (None, 15)           0           activation_414[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_438 (Lambda)             (None, 15)           0           activation_416[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_440 (Lambda)             (None, 15)           0           activation_418[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_442 (Lambda)             (None, 15)           0           activation_420[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_401 (Activation)     (None, 15)           0           lambda_422[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_403 (Activation)     (None, 15)           0           lambda_424[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_405 (Activation)     (None, 15)           0           lambda_426[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_407 (Activation)     (None, 15)           0           lambda_428[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_409 (Activation)     (None, 15)           0           lambda_430[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_411 (Activation)     (None, 15)           0           lambda_432[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_413 (Activation)     (None, 15)           0           lambda_434[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_415 (Activation)     (None, 15)           0           lambda_436[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_417 (Activation)     (None, 15)           0           lambda_438[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_419 (Activation)     (None, 15)           0           lambda_440[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_421 (Activation)     (None, 15)           0           lambda_442[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_423 (Lambda)             (None,)              0           activation_401[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_425 (Lambda)             (None,)              0           activation_403[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_427 (Lambda)             (None,)              0           activation_405[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_429 (Lambda)             (None,)              0           activation_407[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_431 (Lambda)             (None,)              0           activation_409[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_433 (Lambda)             (None,)              0           activation_411[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_435 (Lambda)             (None,)              0           activation_413[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_437 (Lambda)             (None,)              0           activation_415[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_439 (Lambda)             (None,)              0           activation_417[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_441 (Lambda)             (None,)              0           activation_419[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_443 (Lambda)             (None,)              0           activation_421[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lambda_444 (Lambda)             (None, 11)           0           lambda_423[0][0]                 \n",
      "                                                                 lambda_425[0][0]                 \n",
      "                                                                 lambda_427[0][0]                 \n",
      "                                                                 lambda_429[0][0]                 \n",
      "                                                                 lambda_431[0][0]                 \n",
      "                                                                 lambda_433[0][0]                 \n",
      "                                                                 lambda_435[0][0]                 \n",
      "                                                                 lambda_437[0][0]                 \n",
      "                                                                 lambda_439[0][0]                 \n",
      "                                                                 lambda_441[0][0]                 \n",
      "                                                                 lambda_443[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_50 (Dense)                (None, 1)            12          lambda_444[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 21,002\n",
      "Trainable params: 21,002\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 1s 582us/step - loss: 422.1236\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 0s 148us/step - loss: 242.0767\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 0s 151us/step - loss: 129.8622\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 0s 147us/step - loss: 65.8162\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 0s 145us/step - loss: 30.3271\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.contrib.models.HBMP()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params['embedding_input_dim'] = 200\n",
    "# model.params['embedding_output_dim'] = 100\n",
    "# model.params['embedding_trainable'] = True\n",
    "# model.params['alpha'] = 0.1\n",
    "# model.params['mlp_num_layers'] = 3\n",
    "# model.params['mlp_num_units'] = [10, 10]\n",
    "# model.params['lstm_num_units'] = 5\n",
    "# model.params['dropout_rate'] = 0.1\n",
    "# model.build()\n",
    "\n",
    "model =mz.models.KNRM()\n",
    "model.params['embedding_input_dim'] =  10000\n",
    "model.params['embedding_output_dim'] =  10\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['kernel_num'] = 11\n",
    "model.params['sigma'] = 0.1\n",
    "model.params['exact_sigma'] = 0.001\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.build()\n",
    "\n",
    "# model.build()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/KNRM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MatchLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_15\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 100)      209900      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_left (LSTM)                (None, 15, 256)      365568      embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_right (LSTM)               (None, 15, 256)      365568      embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_445 (Lambda)             (None, 15, 256)      0           lstm_left[0][0]                  \n",
      "                                                                 lstm_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)    (None, 30, 256)      0           lambda_445[0][0]                 \n",
      "                                                                 lstm_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_merge (LSTM)               (None, 512)          1574912     concatenate_11[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, 512)          0           lstm_merge[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_55 (Dense)                (None, 200)          102600      dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_56 (Dense)                (None, 1)            201         dense_55[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,618,749\n",
      "Trainable params: 2,618,749\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 59s 29ms/step - loss: 0.1630\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 57s 29ms/step - loss: 0.1583\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 57s 29ms/step - loss: 0.1579\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 58s 29ms/step - loss: 0.1592\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 59s 29ms/step - loss: 0.1584\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.contrib.models.HBMP()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params['embedding_input_dim'] = 200\n",
    "# model.params['embedding_output_dim'] = 100\n",
    "# model.params['embedding_trainable'] = True\n",
    "# model.params['alpha'] = 0.1\n",
    "# model.params['mlp_num_layers'] = 3\n",
    "# model.params['mlp_num_units'] = [10, 10]\n",
    "# model.params['lstm_num_units'] = 5\n",
    "# model.params['dropout_rate'] = 0.1\n",
    "# model.build()\n",
    "\n",
    "model =mz.contrib.models.MatchLSTM()\n",
    "# model = MatchLSTM()\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.params['embedding_input_dim'] = 10000\n",
    "model.params['embedding_output_dim'] = 100\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['fc_num_units'] = 200\n",
    "model.params['lstm_num_units'] = 256\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/MatchLSTM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.models.naive.Naive,\n",
       " matchzoo.models.dssm.DSSM,\n",
       " matchzoo.models.cdssm.CDSSM,\n",
       " matchzoo.models.dense_baseline.DenseBaseline,\n",
       " matchzoo.models.arci.ArcI,\n",
       " matchzoo.models.arcii.ArcII,\n",
       " matchzoo.models.match_pyramid.MatchPyramid,\n",
       " matchzoo.models.knrm.KNRM,\n",
       " matchzoo.models.duet.DUET,\n",
       " matchzoo.models.drmmtks.DRMMTKS,\n",
       " matchzoo.models.drmm.DRMM,\n",
       " matchzoo.models.anmm.ANMM,\n",
       " matchzoo.models.mvlstm.MVLSTM,\n",
       " matchzoo.contrib.models.match_lstm.MatchLSTM,\n",
       " matchzoo.contrib.models.match_srnn.MatchSRNN,\n",
       " matchzoo.contrib.models.hbmp.HBMP,\n",
       " matchzoo.contrib.models.esim.ESIM,\n",
       " matchzoo.contrib.models.bimpm.BiMPM,\n",
       " matchzoo.contrib.models.diin.DIIN,\n",
       " matchzoo.models.conv_knrm.ConvKNRM]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.models.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MatchSRNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_16\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 300)      629700      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "matching_tensor_layer_1 (Matchi (None, 4, 15, 15)    360000      embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "spatial_gru_1 (SpatialGRU)      (None, 10)           2800        matching_tensor_layer_1[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)            (None, 10)           0           spatial_gru_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_57 (Dense)                (None, 1)            11          dropout_16[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 992,511\n",
      "Trainable params: 992,511\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 32s 16ms/step - loss: 0.1644\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 32s 16ms/step - loss: 0.1575\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 32s 16ms/step - loss: 0.1235\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 30s 15ms/step - loss: 0.0139\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 31s 15ms/step - loss: 0.0035\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.contrib.models.HBMP()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params['embedding_input_dim'] = 200\n",
    "# model.params['embedding_output_dim'] = 100\n",
    "# model.params['embedding_trainable'] = True\n",
    "# model.params['alpha'] = 0.1\n",
    "# model.params['mlp_num_layers'] = 3\n",
    "# model.params['mlp_num_units'] = [10, 10]\n",
    "# model.params['lstm_num_units'] = 5\n",
    "# model.params['dropout_rate'] = 0.1\n",
    "# model.build()\n",
    "\n",
    "model =mz.contrib.models.MatchSRNN()\n",
    "model.params['channels'] = 4\n",
    "model.params['units'] = 10\n",
    "model.params['dropout_rate'] = 0.0\n",
    "model.params['direction'] = 'lt'\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/MatchSRNN.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.models.naive.Naive,\n",
       " matchzoo.models.dssm.DSSM,\n",
       " matchzoo.models.cdssm.CDSSM,\n",
       " matchzoo.models.dense_baseline.DenseBaseline,\n",
       " matchzoo.models.arci.ArcI,\n",
       " matchzoo.models.arcii.ArcII,\n",
       " matchzoo.models.match_pyramid.MatchPyramid,\n",
       " matchzoo.models.knrm.KNRM,\n",
       " matchzoo.models.duet.DUET,\n",
       " matchzoo.models.drmmtks.DRMMTKS,\n",
       " matchzoo.models.drmm.DRMM,\n",
       " matchzoo.models.anmm.ANMM,\n",
       " matchzoo.models.mvlstm.MVLSTM,\n",
       " matchzoo.contrib.models.match_lstm.MatchLSTM,\n",
       " matchzoo.contrib.models.match_srnn.MatchSRNN,\n",
       " matchzoo.contrib.models.hbmp.HBMP,\n",
       " matchzoo.contrib.models.esim.ESIM,\n",
       " matchzoo.contrib.models.bimpm.BiMPM,\n",
       " matchzoo.contrib.models.diin.DIIN,\n",
       " matchzoo.models.conv_knrm.ConvKNRM]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.models.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MVLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_17\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 300)      629700      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_9 (Bidirectional) (None, 15, 64)       85248       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_10 (Bidirectional (None, 15, 64)       85248       embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dot_33 (Dot)                    (None, 15, 15)       0           bidirectional_9[0][0]            \n",
      "                                                                 bidirectional_10[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "reshape_10 (Reshape)            (None, 225)          0           dot_33[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_446 (Lambda)             (None, 50)           0           reshape_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_58 (Dense)                (None, 20)           1020        lambda_446[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_59 (Dense)                (None, 20)           420         dense_58[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_60 (Dense)                (None, 10)           210         dense_59[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)            (None, 10)           0           dense_60[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_61 (Dense)                (None, 1)            11          dropout_17[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 801,857\n",
      "Trainable params: 801,857\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 9s 5ms/step - loss: 0.1712\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 7s 3ms/step - loss: 0.1639\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 7s 4ms/step - loss: 0.1616\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 8s 4ms/step - loss: 0.1597\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 8s 4ms/step - loss: 0.1576\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.contrib.models.HBMP()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params['embedding_input_dim'] = 200\n",
    "# model.params['embedding_output_dim'] = 100\n",
    "# model.params['embedding_trainable'] = True\n",
    "# model.params['alpha'] = 0.1\n",
    "# model.params['mlp_num_layers'] = 3\n",
    "# model.params['mlp_num_units'] = [10, 10]\n",
    "# model.params['lstm_num_units'] = 5\n",
    "# model.params['dropout_rate'] = 0.1\n",
    "# model.build()\n",
    "\n",
    "model =mz.models.MVLSTM()\n",
    "model.params['lstm_units'] = 32\n",
    "model.params['top_k'] = 50\n",
    "model.params['mlp_num_layers'] = 2\n",
    "model.params['mlp_num_units'] = 20\n",
    "model.params['mlp_num_fan_out'] = 10\n",
    "model.params['mlp_activation_func'] = 'relu'\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/MVLSTM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BiMPM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "Ranking Task\n",
      "Model: \"model_20\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 300)      629700      text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)            (None, 15, 300)      0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_22 (Dropout)            (None, 15, 300)      0           embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_11 (Bidirectional [(None, 15, 8), (Non 9760        dropout_21[0][0]                 \n",
      "                                                                 dropout_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multi_perspective_layer_1 (Mult (None, 15, 15)       24          bidirectional_11[0][0]           \n",
      "                                                                 bidirectional_11[0][1]           \n",
      "                                                                 bidirectional_11[0][2]           \n",
      "                                                                 bidirectional_11[0][3]           \n",
      "                                                                 bidirectional_11[0][4]           \n",
      "                                                                 bidirectional_11[1][0]           \n",
      "                                                                 bidirectional_11[1][1]           \n",
      "                                                                 bidirectional_11[1][2]           \n",
      "                                                                 bidirectional_11[1][3]           \n",
      "                                                                 bidirectional_11[1][4]           \n",
      "                                                                 bidirectional_11[1][0]           \n",
      "                                                                 bidirectional_11[1][1]           \n",
      "                                                                 bidirectional_11[1][2]           \n",
      "                                                                 bidirectional_11[1][3]           \n",
      "                                                                 bidirectional_11[1][4]           \n",
      "                                                                 bidirectional_11[0][0]           \n",
      "                                                                 bidirectional_11[0][1]           \n",
      "                                                                 bidirectional_11[0][2]           \n",
      "                                                                 bidirectional_11[0][3]           \n",
      "                                                                 bidirectional_11[0][4]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_23 (Dropout)            (None, 15, 15)       0           multi_perspective_layer_1[0][0]  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "__________________________________________________________________________________________________\n",
      "dropout_24 (Dropout)            (None, 15, 15)       0           multi_perspective_layer_1[1][0]  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_12 (Bidirectional (None, 8)            640         dropout_23[0][0]                 \n",
      "                                                                 dropout_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_12 (Concatenate)    (None, 16)           0           bidirectional_12[0][0]           \n",
      "                                                                 bidirectional_12[1][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_73 (Dense)                (None, 4)            68          concatenate_12[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_74 (Dense)                (None, 4)            20          dense_73[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_75 (Dense)                (None, 1)            5           dense_74[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 640,217\n",
      "Trainable params: 640,217\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 11s 5ms/step - loss: 0.1615\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 6s 3ms/step - loss: 0.1564\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 5s 3ms/step - loss: 0.1245\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 5s 3ms/step - loss: 0.0094\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 6s 3ms/step - loss: 0.0036\n",
      "{mean_average_precision(0.0): 0.18562874251497005}\n",
      "保存成功\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "print(mz.__version__)\n",
    "import tensorflow.keras as K\n",
    "\n",
    "### 定义任务，包含两种，一个是Ranking，一个是classification\n",
    "task = mz.tasks.Ranking()\n",
    "print(task)\n",
    "\n",
    "\n",
    "\n",
    "# model = mz.contrib.models.HBMP()\n",
    "# model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.params['embedding_input_dim'] = 200\n",
    "# model.params['embedding_output_dim'] = 100\n",
    "# model.params['embedding_trainable'] = True\n",
    "# model.params['alpha'] = 0.1\n",
    "# model.params['mlp_num_layers'] = 3\n",
    "# model.params['mlp_num_units'] = [10, 10]\n",
    "# model.params['lstm_num_units'] = 5\n",
    "# model.params['dropout_rate'] = 0.1\n",
    "# model.build()\n",
    "\n",
    "model = mz.contrib.models.BiMPM()\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "# model.build()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params.completed()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()\n",
    "\n",
    "\n",
    "### 训练, 评估, 预测\n",
    "x, y = train_pack_processed .unpack()\n",
    "# test_x, test_y = test_processed.unpack()\n",
    "# model.fit(x , y,batch_size=32, epochs=5)\n",
    "model.fit(x , y,batch_size=32, epochs=5)\n",
    "print(model.evaluate(dev_x,dev_y))\n",
    "# model.predict(test_x)\n",
    "\n",
    "### 保存模型\n",
    "model.save('./outputs/model/BiMPM.h5')\n",
    "print('保存成功')\n",
    "# loaded_model = mz.load_model('./outputs/model/DenseBaseline-model.h5')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
